require 'config/boot'
require 'config/environment'
require 'lib/simple-json'

cashe_size = ENV['cashe_size'].nil? ? 1000 : ENV['cashe_size'].to_i
task :count do
  json_parser = JsonParser.new
  size = Corpus.count
  limit = cashe_size
  repeat_times = (size / limit) + 1
  repeat_times.times{|i|
    segments =  Hash.new {|hash, key| hash[key] = 0}
    p "cashe segments at #{i}/#{repeat_times}"
    Corpus.find(:all,
      :select =>"id,data,json",
      :limit => limit,
      :offset => limit * i
    ).each{|sentence|
      json_data = json_parser.parse(sentence.json)
      sentence.data.split("\t").each_with_index{|data,index|
        json_data[index]["data"] = data
        segments[json_data[index]] += 1 
      }
    }
    p "regist segments at #{i}/#{repeat_times}"
    segments.each{|segment,size|
      record = Segment.find(:first,:conditions => segment)
      record = Segment.create(segment) if record.nil?
      record.update_attribute(:size, record.size + size)
    }
  }
end